import csv
import requests
import bs4
import re
from threading import Thread,BoundedSemaphore,Lock,Semaphore


def getBidRecoed(assetsCode):
    headers = {'Host': 'www.dscq.com','accept': 'application/json, text/javascript, */*; q=0.01',
               'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36',
               'x-requested-with': 'XMLHttpRequest','sec-fetch-site': 'same-origin','sec-fetch-mode': 'cors',
               'sec-fetch-dest': 'empty',
               'referer': 'https://www.dscq.com/bidDetails/e46fdbc8ad8e4a948a19f83679c4dba9.shtml',
               'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',}
    params = (('assetsCode',f'{assetsCode}'),('pageSize','20'),('currentNumber','1'),('_','1612398554350'),)
    # print(params)
    response = requests.get( 'https://www.dscq.com/hall/records.shtml',headers = headers,params = params,
        cookies = cookies )
    return response.json()[ "total" ]


def getInfo(ex,s):
    s.acquire()
    if ex not in succcess:
        # pass
        # ex = "0bb24422e52a43478b995189e088fff3"
        # ex = "525096542a2440dfbc760babd4fb1a8b"
        finalUrl = f'http://www.hebaee.cn/assets/gyzc-detail/{ex}.html'
        print(finalUrl)

        response = requests.get( finalUrl,headers = headers,cookies = cookies )
        soup = bs4.BeautifulSoup( response.text,"html.parser" )
        if "竞拍目录" in soup.text:
            s.release()
            return  None

        if "已成交" in soup.text:
            chengjiao = 1
        else:
            chengjiao = 0

        assetsCode = re.findall( 'assetsCode = "(.+)"',response.text )[ 0 ]
        bidNum = getBidRecoed( assetsCode )
        try:
            gpj = float(
                soup.find( "span",class_ = "gpj-price" ).text.replace( "\r","" ).replace( "\n","" ).replace( "¥",
                    "" ).replace( " ","" ).replace( "元/万吨","" ).replace( ",","" ) )
        except:
            gpj = 0
        try:
            bzj = float(
                soup.find( "span",class_ = "bzj-price" ).text.replace( "\r","" ).replace( "\n","" ).replace( "¥",
                    "" ).replace( " ","" ).replace( ",","" ) )
        except:
            bzj = 0
        try:
            pgj = float(
                soup.find( "span",class_ = "pgj-price" ).text.replace( "\r","" ).replace( "\n","" ).replace( "¥",
                    "" ).replace( " ","" ).replace( "——","0" ).replace( ",","" ) )
        except:
            pgj = 0
        print( response.url )
        try:
            yj = gpj - pgj
            yjl = yj / pgj
        except:
            yj = "-"
            yjl = "-"
            pass
        outRow = [ ex,chengjiao,gpj,bzj,pgj,yj,yjl,assetsCode,bidNum ]
        with lock:
            with open( "./idPrice.csv",encoding = "utf-8-sig",mode = "a",errors = "ignore",newline = "" ) as f:
                writer = csv.writer( f )
                writer.writerow( outRow )
            print( outRow )
    else:
        print( f'{ex} is REPEATED !' )
    s.release()


def main():
    idPool = [ ]
    with open( './index.csv',mode = "r",encoding = "utf-8" ) as f:
        reader = csv.reader( f )
        for row in reader:
            idPool.append( row[ 13 ] )

    n = list( range( len( (idPool) ) ) )
    semaphore = Semaphore( 10 )
    thread = [ ]
    for i in n:
        t = Thread( target = getInfo,args = (idPool[ i ],semaphore) )
        thread.append( t )
    for i in n:
        thread[ i ].start()
    for i in n:
        thread[ i ].join()


if __name__ == "__main__":
    cookies = {'Hm_lvt_3e46284be460d08cb4b834b45a9b3bb6': '1612361894',
               'Hm_lvt_b9a324c2fdf4142db72952069584f3d0': '1612361894',
               'SERVERID': '33ddf3ccd93c8e477e529224884a682f|1612365236|1612363673',
               'Hm_lpvt_b9a324c2fdf4142db72952069584f3d0': '1612365235',
               'Hm_lpvt_3e46284be460d08cb4b834b45a9b3bb6': '1612365236',}

    headers = {'Host': 'www.hebaee.cn','Upgrade-Insecure-Requests': '1',
               'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36',
               'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
               'Referer': 'http://www.hebaee.cn/search.html?&province=%E6%B2%B3%E5%8C%97&city=&assetsTypeParent=&assetsTypeLeaf=&state=%E5%B7%B2%E7%BB%93%E6%9D%9F&orderBy=&order=&keyWords=&minPrice=&maxPrice=&start=8',
               'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',}

    succcess = [ ]
    try:
        with open( './idPrice.csv',mode = "r",encoding = "utf-8" ) as f:
            reader = csv.reader( f )
            for row in reader:
                succcess.append( row[ 0 ] )
    except Exception as e:
        print( e )
    lock = Lock()
    main()
